from langchain_community.document_loaders import DirectoryLoader, TextLoader
from pathlib import Path
from typing import List
from langchain.docstore.document import Document

class DocumentLoader:
    def __init__(self, documents_dir: Path):
        self.documents_dir = documents_dir
        
    def load_documents(self) -> List[Document]:
        """加载文档"""
        # 确保目录存在
        if not self.documents_dir.exists():
            raise ValueError(f"Documents directory {self.documents_dir} does not exist")
            
        # 使用 DirectoryLoader 加载文本文件
        loader = DirectoryLoader(
            str(self.documents_dir),
            glob="**/*.txt",  # 加载所有 .txt 文件
            loader_cls=TextLoader,
            show_progress=True
        )
        
        documents = loader.load()
        
        # 添加调试信息
        print(f"Loading documents from: {self.documents_dir}")
        print(f"Found {len(documents)} documents")
        
        return documents 